In [1]:
import tensorflow as tf
import numpy as np
import pandas as pd

import autoencoder_train
import network
import utils
In [2]:
datadir = "/home/cjw/disk2cjw/Data/cyto/"
#datadir = "/home/cjw/disk2cjw/Data/yeast/"
mmfilename = datadir + "Fish6_15_cjw_mean_zero.mm"
#mmfilename = datadir + "mmplate15-1.mm"
mmheader = np.memmap(mmfilename, dtype="int32", mode='r',
                    shape=(4,))

header_shape = mmheader.shape
print(header_shape, mmheader)
xshape = [mmheader[0], mmheader[1], mmheader[2], mmheader[3]]
#xshape[0] = 2400
xshape = tuple(xshape)
del mmheader
print(xshape)

m3 = np.memmap(mmfilename, dtype='float32', offset=128,
              mode='r', shape=xshape)
(4,) [10000    64    64     1]
(10000, 64, 64, 1)
In [3]:
n = xshape[0]
ids =  range(n)
file = n*['Fish6_15_cjw_64.tif']
fid = range(n)
mmfile = n*['Fish6_15_cjw_64.tif.mm']
plate = n*[0]
row = n*[0]
column = n*[0]
field = n*[0]
yc = n*[32]
xc = n*[32]
well = n*[0]

p_df = pd.DataFrame({'id':ids, 'fid':fid, 'file':file, 'mmfile':mmfile, 'plate':plate,
                 'row':row, 'column':column, 'field':field,
                  'yc':yc, 'xc':xc, 'well':well})

p_df.head()
Out[3]:
column fid field file id mmfile plate row well xc yc
0 0 0 0 Fish6_15_cjw_64.tif 0 Fish6_15_cjw_64.tif.mm 0 0 0 32 32
1 0 1 0 Fish6_15_cjw_64.tif 1 Fish6_15_cjw_64.tif.mm 0 0 0 32 32
2 0 2 0 Fish6_15_cjw_64.tif 2 Fish6_15_cjw_64.tif.mm 0 0 0 32 32
3 0 3 0 Fish6_15_cjw_64.tif 3 Fish6_15_cjw_64.tif.mm 0 0 0 32 32
4 0 4 0 Fish6_15_cjw_64.tif 4 Fish6_15_cjw_64.tif.mm 0 0 0 32 32
In [ ]:
 
In [4]:
p_mmdict = {"Fish6_15_cjw_64.tif.mm": m3}
#             "plate04_all.mm": m4}

p_width = 64
p_height = 64
p_nchannels = 1
p_channels = [0]
p_nepochs = 60
p_batchsize = 32
p_learning_rate = 0.00005  #.00005
p_restore = False
p_latent_size = 128

# enc_sizes = [(8, 7), (16, 5), (32, 3), (64, 3)]
enc_sizes = [(64, 5), (128, 3), (256,3), (512,3)] #, (64, 3), (128,3)] #, (32, 3), (64, 3)]
#enc_sizes = [(16, 5), (32, 3), (64,3), (128,3)] #, (64, 3), (128,3)] #, (32, 3), (64, 3)]
dec_sizes = list(reversed(enc_sizes))
#dec_sizes.append((p_nchannels,7))

params = dict()

params['width'] = p_width
params['height'] = p_height
params['nchannels'] = p_nchannels
params['channels'] = p_channels
params['nepochs'] = p_nepochs
params['batchsize'] = p_batchsize
params['learning_rate'] = p_learning_rate
params['restore'] = p_restore
params['latent_size'] = p_latent_size
params['droprate'] = 0.75

params['enc_sizes'] = enc_sizes
params['dec_sizes'] = dec_sizes

autoencoder_train.train(p_mmdict, p_df, params, 100)
Encoder True
[<tf.Tensor 'Placeholder:0' shape=(?, 64, 64, 1) dtype=float32>, <tf.Tensor 'dropout/mul:0' shape=(?, 32, 32, 64) dtype=float32>, <tf.Tensor 'dropout_1/mul:0' shape=(?, 16, 16, 128) dtype=float32>, <tf.Tensor 'dropout_2/mul:0' shape=(?, 8, 8, 256) dtype=float32>, <tf.Tensor 'dropout_3/mul:0' shape=(?, 4, 4, 512) dtype=float32>] Tensor("latent_space/BiasAdd:0", shape=(?, 128), dtype=float32)
isize:  4 64
[<tf.Tensor 'dropout_4/mul:0' shape=(?, 8192) dtype=float32>, <tf.Tensor 'Reshape:0' shape=(?, 4, 4, 512) dtype=float32>, <tf.Tensor 'dropout_5/mul:0' shape=(?, 8, 8, 256) dtype=float32>, <tf.Tensor 'dropout_6/mul:0' shape=(?, 16, 16, 128) dtype=float32>, <tf.Tensor 'dropout_7/mul:0' shape=(?, 32, 32, 64) dtype=float32>] Tensor("decoder_out/BiasAdd:0", shape=(?, 64, 64, 1), dtype=float32)
Epoch:  0 Iteration:  0 Loss:  7.32676
Epoch:  0 Iteration:  100 Loss:  7.71603
Epoch:  0 Iteration:  200 Loss:  10.191
Epoch:  0 Iteration:  300 Loss:  9.70606
Epoch:  1 Iteration:  0 Loss:  7.05811
Epoch:  1 Iteration:  100 Loss:  7.27117
Epoch:  1 Iteration:  200 Loss:  8.36355
Epoch:  1 Iteration:  300 Loss:  7.45515
Epoch:  2 Iteration:  0 Loss:  4.84645
Epoch:  2 Iteration:  100 Loss:  5.10158
Epoch:  2 Iteration:  200 Loss:  5.89903
Epoch:  2 Iteration:  300 Loss:  5.92645
Epoch:  3 Iteration:  0 Loss:  3.76526
Epoch:  3 Iteration:  100 Loss:  4.16914
Epoch:  3 Iteration:  200 Loss:  4.73699
Epoch:  3 Iteration:  300 Loss:  5.09088
Epoch:  4 Iteration:  0 Loss:  3.30279
Epoch:  4 Iteration:  100 Loss:  3.70288
Epoch:  4 Iteration:  200 Loss:  4.35059
Epoch:  4 Iteration:  300 Loss:  4.53015
Epoch:  5 Iteration:  0 Loss:  3.05782
Epoch:  5 Iteration:  100 Loss:  3.39187
Epoch:  5 Iteration:  200 Loss:  3.91876
Epoch:  5 Iteration:  300 Loss:  4.28351
Epoch:  6 Iteration:  0 Loss:  2.85756
Epoch:  6 Iteration:  100 Loss:  3.0741
Epoch:  6 Iteration:  200 Loss:  3.60831
Epoch:  6 Iteration:  300 Loss:  4.06641
Epoch:  7 Iteration:  0 Loss:  2.62512
Epoch:  7 Iteration:  100 Loss:  2.88292
Epoch:  7 Iteration:  200 Loss:  3.4058
Epoch:  7 Iteration:  300 Loss:  3.77168
Epoch:  8 Iteration:  0 Loss:  2.56875
Epoch:  8 Iteration:  100 Loss:  2.79014
Epoch:  8 Iteration:  200 Loss:  3.40121
Epoch:  8 Iteration:  300 Loss:  3.64101
Epoch:  9 Iteration:  0 Loss:  2.33487
Epoch:  9 Iteration:  100 Loss:  2.73201
Epoch:  9 Iteration:  200 Loss:  3.18109
Epoch:  9 Iteration:  300 Loss:  3.48075
Epoch:  10 Iteration:  0 Loss:  2.35941
Epoch:  10 Iteration:  100 Loss:  2.57249
Epoch:  10 Iteration:  200 Loss:  3.04349
Epoch:  10 Iteration:  300 Loss:  3.41252
Epoch:  11 Iteration:  0 Loss:  2.24347
Epoch:  11 Iteration:  100 Loss:  2.50488
Epoch:  11 Iteration:  200 Loss:  2.90322
Epoch:  11 Iteration:  300 Loss:  3.31153
Epoch:  12 Iteration:  0 Loss:  2.13592
Epoch:  12 Iteration:  100 Loss:  2.42005
Epoch:  12 Iteration:  200 Loss:  2.79729
Epoch:  12 Iteration:  300 Loss:  3.11014
Epoch:  13 Iteration:  0 Loss:  2.12132
Epoch:  13 Iteration:  100 Loss:  2.4669
Epoch:  13 Iteration:  200 Loss:  2.74425
Epoch:  13 Iteration:  300 Loss:  3.06295
Epoch:  14 Iteration:  0 Loss:  2.04893
Epoch:  14 Iteration:  100 Loss:  2.36483
Epoch:  14 Iteration:  200 Loss:  2.61718
Epoch:  14 Iteration:  300 Loss:  3.0116
Epoch:  15 Iteration:  0 Loss:  2.03941
Epoch:  15 Iteration:  100 Loss:  2.28892
Epoch:  15 Iteration:  200 Loss:  2.6318
Epoch:  15 Iteration:  300 Loss:  2.91555
Epoch:  16 Iteration:  0 Loss:  1.95916
Epoch:  16 Iteration:  100 Loss:  2.25482
Epoch:  16 Iteration:  200 Loss:  2.45606
Epoch:  16 Iteration:  300 Loss:  2.84401
Epoch:  17 Iteration:  0 Loss:  1.90808
Epoch:  17 Iteration:  100 Loss:  2.18467
Epoch:  17 Iteration:  200 Loss:  2.36682
Epoch:  17 Iteration:  300 Loss:  2.69328
Epoch:  18 Iteration:  0 Loss:  1.86353
Epoch:  18 Iteration:  100 Loss:  2.22555
Epoch:  18 Iteration:  200 Loss:  2.32175
Epoch:  18 Iteration:  300 Loss:  2.70058
Epoch:  19 Iteration:  0 Loss:  1.8665
Epoch:  19 Iteration:  100 Loss:  2.1499
Epoch:  19 Iteration:  200 Loss:  2.33119
Epoch:  19 Iteration:  300 Loss:  2.61786
Epoch:  20 Iteration:  0 Loss:  1.83822
Epoch:  20 Iteration:  100 Loss:  2.12328
Epoch:  20 Iteration:  200 Loss:  2.25107
Epoch:  20 Iteration:  300 Loss:  2.60055
Epoch:  21 Iteration:  0 Loss:  1.79189
Epoch:  21 Iteration:  100 Loss:  2.06962
Epoch:  21 Iteration:  200 Loss:  2.22567
Epoch:  21 Iteration:  300 Loss:  2.52725
Epoch:  22 Iteration:  0 Loss:  1.75192
Epoch:  22 Iteration:  100 Loss:  2.08387
Epoch:  22 Iteration:  200 Loss:  2.18012
Epoch:  22 Iteration:  300 Loss:  2.48223
Epoch:  23 Iteration:  0 Loss:  1.64772
Epoch:  23 Iteration:  100 Loss:  1.98585
Epoch:  23 Iteration:  200 Loss:  2.12295
Epoch:  23 Iteration:  300 Loss:  2.53055
Epoch:  24 Iteration:  0 Loss:  1.69009
Epoch:  24 Iteration:  100 Loss:  1.99458
Epoch:  24 Iteration:  200 Loss:  2.13499
Epoch:  24 Iteration:  300 Loss:  2.39252
Epoch:  25 Iteration:  0 Loss:  1.68303
Epoch:  25 Iteration:  100 Loss:  1.94506
Epoch:  25 Iteration:  200 Loss:  2.13162
Epoch:  25 Iteration:  300 Loss:  2.32425
Epoch:  26 Iteration:  0 Loss:  1.62746
Epoch:  26 Iteration:  100 Loss:  1.93436
Epoch:  26 Iteration:  200 Loss:  2.01816
Epoch:  26 Iteration:  300 Loss:  2.42488
Epoch:  27 Iteration:  0 Loss:  1.54012
Epoch:  27 Iteration:  100 Loss:  1.89031
Epoch:  27 Iteration:  200 Loss:  2.00002
Epoch:  27 Iteration:  300 Loss:  2.42006
Epoch:  28 Iteration:  0 Loss:  1.59259
Epoch:  28 Iteration:  100 Loss:  1.9628
Epoch:  28 Iteration:  200 Loss:  2.05088
Epoch:  28 Iteration:  300 Loss:  2.36159
Epoch:  29 Iteration:  0 Loss:  1.52939
Epoch:  29 Iteration:  100 Loss:  1.82381
Epoch:  29 Iteration:  200 Loss:  1.95561
Epoch:  29 Iteration:  300 Loss:  2.31217
Epoch:  30 Iteration:  0 Loss:  1.5309
Epoch:  30 Iteration:  100 Loss:  1.85837
Epoch:  30 Iteration:  200 Loss:  1.9552
Epoch:  30 Iteration:  300 Loss:  2.26036
Epoch:  31 Iteration:  0 Loss:  1.49064
Epoch:  31 Iteration:  100 Loss:  1.78112
Epoch:  31 Iteration:  200 Loss:  1.88147
Epoch:  31 Iteration:  300 Loss:  2.23564
Epoch:  32 Iteration:  0 Loss:  1.46833
Epoch:  32 Iteration:  100 Loss:  1.87365
Epoch:  32 Iteration:  200 Loss:  1.8947
Epoch:  32 Iteration:  300 Loss:  2.18398
Epoch:  33 Iteration:  0 Loss:  1.45646
Epoch:  33 Iteration:  100 Loss:  1.75203
Epoch:  33 Iteration:  200 Loss:  1.87789
Epoch:  33 Iteration:  300 Loss:  2.18324
Epoch:  34 Iteration:  0 Loss:  1.47273
Epoch:  34 Iteration:  100 Loss:  1.76651
Epoch:  34 Iteration:  200 Loss:  1.80627
Epoch:  34 Iteration:  300 Loss:  2.15333
Epoch:  35 Iteration:  0 Loss:  1.38676
Epoch:  35 Iteration:  100 Loss:  1.78734
Epoch:  35 Iteration:  200 Loss:  1.8563
Epoch:  35 Iteration:  300 Loss:  2.09381
Epoch:  36 Iteration:  0 Loss:  1.4121
Epoch:  36 Iteration:  100 Loss:  1.74267
Epoch:  36 Iteration:  200 Loss:  1.80439
Epoch:  36 Iteration:  300 Loss:  2.10339
Epoch:  37 Iteration:  0 Loss:  1.34795
Epoch:  37 Iteration:  100 Loss:  1.72407
Epoch:  37 Iteration:  200 Loss:  1.75947
Epoch:  37 Iteration:  300 Loss:  2.10681
Epoch:  38 Iteration:  0 Loss:  1.33238
Epoch:  38 Iteration:  100 Loss:  1.70561
Epoch:  38 Iteration:  200 Loss:  1.80167
Epoch:  38 Iteration:  300 Loss:  2.12591
Epoch:  39 Iteration:  0 Loss:  1.33787
Epoch:  39 Iteration:  100 Loss:  1.71656
Epoch:  39 Iteration:  200 Loss:  1.74068
Epoch:  39 Iteration:  300 Loss:  2.07154
Epoch:  40 Iteration:  0 Loss:  1.29816
Epoch:  40 Iteration:  100 Loss:  1.73858
Epoch:  40 Iteration:  200 Loss:  1.72156
Epoch:  40 Iteration:  300 Loss:  1.964
Epoch:  41 Iteration:  0 Loss:  1.34141
Epoch:  41 Iteration:  100 Loss:  1.68165
Epoch:  41 Iteration:  200 Loss:  1.74772
Epoch:  41 Iteration:  300 Loss:  2.03914
Epoch:  42 Iteration:  0 Loss:  1.31132
Epoch:  42 Iteration:  100 Loss:  1.67285
Epoch:  42 Iteration:  200 Loss:  1.72362
Epoch:  42 Iteration:  300 Loss:  2.00983
Epoch:  43 Iteration:  0 Loss:  1.32811
Epoch:  43 Iteration:  100 Loss:  1.65424
Epoch:  43 Iteration:  200 Loss:  1.65666
Epoch:  43 Iteration:  300 Loss:  1.97727
Epoch:  44 Iteration:  0 Loss:  1.26686
Epoch:  44 Iteration:  100 Loss:  1.67159
Epoch:  44 Iteration:  200 Loss:  1.65766
Epoch:  44 Iteration:  300 Loss:  1.93742
Epoch:  45 Iteration:  0 Loss:  1.23347
Epoch:  45 Iteration:  100 Loss:  1.65962
Epoch:  45 Iteration:  200 Loss:  1.67777
Epoch:  45 Iteration:  300 Loss:  1.95228
Epoch:  46 Iteration:  0 Loss:  1.26836
Epoch:  46 Iteration:  100 Loss:  1.58797
Epoch:  46 Iteration:  200 Loss:  1.64444
Epoch:  46 Iteration:  300 Loss:  1.88919
Epoch:  47 Iteration:  0 Loss:  1.23136
Epoch:  47 Iteration:  100 Loss:  1.60299
Epoch:  47 Iteration:  200 Loss:  1.67795
Epoch:  47 Iteration:  300 Loss:  1.9187
Epoch:  48 Iteration:  0 Loss:  1.17759
Epoch:  48 Iteration:  100 Loss:  1.60495
Epoch:  48 Iteration:  200 Loss:  1.66314
Epoch:  48 Iteration:  300 Loss:  1.84234
Epoch:  49 Iteration:  0 Loss:  1.16915
Epoch:  49 Iteration:  100 Loss:  1.58377
Epoch:  49 Iteration:  200 Loss:  1.59217
Epoch:  49 Iteration:  300 Loss:  1.89985
Epoch:  50 Iteration:  0 Loss:  1.22911
Epoch:  50 Iteration:  100 Loss:  1.57918
Epoch:  50 Iteration:  200 Loss:  1.59204
Epoch:  50 Iteration:  300 Loss:  1.83571
Epoch:  51 Iteration:  0 Loss:  1.19847
Epoch:  51 Iteration:  100 Loss:  1.5465
Epoch:  51 Iteration:  200 Loss:  1.55914
Epoch:  51 Iteration:  300 Loss:  1.86831
Epoch:  52 Iteration:  0 Loss:  1.17211
Epoch:  52 Iteration:  100 Loss:  1.58046
Epoch:  52 Iteration:  200 Loss:  1.58504
Epoch:  52 Iteration:  300 Loss:  1.85275
Epoch:  53 Iteration:  0 Loss:  1.15916
Epoch:  53 Iteration:  100 Loss:  1.51248
Epoch:  53 Iteration:  200 Loss:  1.55041
Epoch:  53 Iteration:  300 Loss:  1.86258
Epoch:  54 Iteration:  0 Loss:  1.12447
Epoch:  54 Iteration:  100 Loss:  1.56373
Epoch:  54 Iteration:  200 Loss:  1.56813
Epoch:  54 Iteration:  300 Loss:  1.84487
Epoch:  55 Iteration:  0 Loss:  1.13033
Epoch:  55 Iteration:  100 Loss:  1.5209
Epoch:  55 Iteration:  200 Loss:  1.55312
Epoch:  55 Iteration:  300 Loss:  1.80406
Epoch:  56 Iteration:  0 Loss:  1.17002
Epoch:  56 Iteration:  100 Loss:  1.51059
Epoch:  56 Iteration:  200 Loss:  1.55412
Epoch:  56 Iteration:  300 Loss:  1.73988
Epoch:  57 Iteration:  0 Loss:  1.14422
Epoch:  57 Iteration:  100 Loss:  1.53509
Epoch:  57 Iteration:  200 Loss:  1.51584
Epoch:  57 Iteration:  300 Loss:  1.80341
Epoch:  58 Iteration:  0 Loss:  1.13692
Epoch:  58 Iteration:  100 Loss:  1.5292
Epoch:  58 Iteration:  200 Loss:  1.53558
Epoch:  58 Iteration:  300 Loss:  1.77583
Epoch:  59 Iteration:  0 Loss:  1.14878
Epoch:  59 Iteration:  100 Loss:  1.47512
Epoch:  59 Iteration:  200 Loss:  1.48086
Epoch:  59 Iteration:  300 Loss:  1.73259
Done
In [ ]: